# !/usr/bin/env python3
# coding=utf8
"""
回测模块会把回测结果写入文件夹,
此脚本会消费文件夹里的文件, 并将其转存到数据库,
"""
import datetime
import json
import logging
import pathlib
import sys
import time
from typing import Any, Dict, List, Set, Tuple, Type, Optional, Union, Callable
import vnpy.tools.database_cta
import vnpy.trader.utility


def consumer() -> Tuple[int, bool]:
    """
    从文件夹消费文件并转存到数据库, 并返回(消费的文件的个数+消费之后的文件夹是不是空了),
    """
    dir_path: pathlib.Path = vnpy.trader.utility.get_folder_path(vnpy.tools.database_cta.CTA_BACKTEST_RESULT_BF_CONTINUE)

    # 暂时认为所有json文件都需要被消费,
    jsons: List[pathlib.Path] = [_ for _ in dir_path.glob("*.json") if _.is_file()]
    jsons.sort()
    files = jsons[:min(len(jsons), 2000)]  # 如果文件夹里有超多的文件, 如果一次性加载所有的文件, 容易撑爆内存和交换空间,

    # 如果能找到消费文件, 就打印日志, 否则不打印日志, 防止日志文件过大,
    files and logging.info(f"dir_path={dir_path}, len_jsons={len(jsons)}, len_files={len(files)},")

    # 内容正常的json文件转换成类对象, 并缓存文件和对象,
    cache: Dict[pathlib.Path, vnpy.tools.database_cta.CtaDumpData] = {}

    for file in files:
        file: pathlib.Path = file

        line: Optional[vnpy.tools.database_cta.CtaDumpData] = None

        try:
            text: bytes = file.read_bytes()
            data: dict = json.loads(s=text)
            assert isinstance(data, dict)
            line = vnpy.tools.database_cta.CtaDumpData.dict2py(data)
        except Exception as ex:
            logging.info(f"file={file}, ex={ex},")
            continue

        cache[file] = line

    files and logging.info(f"len_cache={len(cache)},")

    # 类对象批量写入数据库,
    lines: List[vnpy.tools.database_cta.CtaDumpData] = [*cache.values()]

    save_ex: Optional[Exception] = None
    try:
        vnpy.tools.database_cta.CtaDumpDataTool.save_to_database_single(cta_dump_data_list=lines)
    except Exception as ex:
        save_ex = ex

    files and logging.info(f"save_ex={save_ex},")

    # 清理成功消费的文件,
    if (save_ex is None):
        for file in cache.keys():
            file.unlink(missing_ok=True)

    # 返回(消费的文件的个数+消费之后的文件夹是不是空了),
    return (len(cache), len(jsons) == len(files)) if (save_ex is None) else (0, False)


if __name__ == "__main__":
    logging.basicConfig(
        filename=f"{__file__}.log", filemode='a', level=logging.INFO,
        format="%(asctime)s (%(process)5d, %(thread)d) %(name)s %(levelname)s - %(message)s",
    )

    logging.info(f"++++++++++ START ++++++++++")

    consume_pit: Optional[datetime.datetime] = None  # point-in-time
    consume_cnt: int = 0

    while True:
        cnt, is_empty = consumer()
        if cnt != 0:
            consume_pit: datetime.datetime = datetime.datetime.now()
            consume_cnt: int = cnt

        message: str = "\r{}, pit={}, cnt={}".format(
            datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
            consume_pit.strftime("%Y-%m-%d %H:%M:%S") if consume_pit else ('_' * 19),
            f"{consume_cnt:<9d}",
        )
        sys.stdout.write(message)

        if is_empty:
            time.sleep(2)
    pass
